#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time   : 2021/3/24 20:56
# @Author : cjw
import requests
import re
import csv
import time


def get_movie_top250():
    """
    获取豆瓣电影top250
    步骤：
    1、拿到页面源代码 requests
    2、通过re来提取想要的有效信息
    :return:
    """
    movies_list = []
    url = 'https://movie.douban.com/top250'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
    }
    # 解析数据, .*?匹配<li>标签下的空格和换行
    obj = re.compile(r'<li>.*?<div class="item">.*?<span class="title">(?P<name>.*?)'
                     r'</span>.*?<p class="">.*?<br>(?P<year>.*?)&nbsp;.*?'
                     r'<span class="rating_num" property="v:average">(?P<score>.*?)</span>.*?'
                     r'<span>(?P<numbers>\d+)人评价</span>.*?<span class="inq">(?P<word>.*?)</span>.*?</li>', re.S)
    for i in range(10):
        data = {
            'start': 25 * i,
            'filter': ''
        }
        resp = requests.get(url, headers=headers, params=data, timeout=10)
        page_text = resp.text
        result = obj.finditer(page_text)
        # 获取匹配到的电影数据
        for rt in result:
            movie_dict = rt.groupdict()
            movie_dict['year'] = movie_dict['year'].strip()
            movies_list.append(movie_dict.values())
        print(data, resp.request.url, len(movies_list))
        resp.close()  # 关闭http长链接
        time.sleep(2)
    print(len(movies_list))
    with open('top250.csv', 'w', encoding='utf-8') as f:
        csv_write = csv.writer(f)
        csv_write.writerows(movies_list)


if __name__ == '__main__':
    get_movie_top250()
